package com.seo.textgen.pos.tagger;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

import org.apache.commons.lang.StringUtils;

import com.alex.utils.text.TextUtils;

public class TaggedTextParser {

	private EndOfSentenceDetector endOfSentenceDetector = new DefaultEndOfSentenceDetector();

	public List<TaggedSentence> parse(String text, String taggedText) throws IOException {
		List<Object> sentenceParts = new ArrayList<Object>();
		List<Token> tokens = new ArrayList<Token>();
		List<TaggedSentence> sentences = new ArrayList<TaggedSentence>();

		int tokenCount = 0;
		int unknownTokenCount = 0;
		// int badSentences = 0;

		// int i = 0;
		int textPos = 0;
		StringTokenizer stringTokenizer = new StringTokenizer(taggedText, "\n");

		while (stringTokenizer.hasMoreTokens()) {
			String[] parts = StringUtils.split(stringTokenizer.nextToken(), "\t");

			String tokenStr;
			String tag;

			if (parts.length == 3) {
				tokenStr = parts[0];
				tag = parts[1];
				if (!Character.isLetter(parts[1].charAt(0))) {
					continue;
				}
				tokenCount++;
				if ("<unknown>".equals(parts[2])) {
					unknownTokenCount++;
				}
			} else {
				tokenStr = parts[0];
				tag = "unknown";
			}

			// if (i++ % 10000 == 0) {
			// System.out.println(textPos + " of " + text.length());
			// System.out.println(Runtime.getRuntime().freeMemory() / 1024 /
			// 1024 + "M");
			// }

			int start = textPos;
			char firstChar = tokenStr.charAt(0);
			while (textPos < text.length()) {
				if (firstChar == text.charAt(textPos)) {
					if (textPos > start) {
						String part = text.substring(start, textPos);
						if (sentenceParts.isEmpty()) {
							part = trimLeft(part);
						}
						if (!part.isEmpty()) {
							sentenceParts.add(part);
						}
					}
					textPos += tokenStr.length();
					break;
				}
				textPos++;
			}
			
			if (sentenceParts.isEmpty() && TextUtils.isEndOfSentence(tokenStr)) {
				continue;
			}

			Token token = new Token(tokenStr, tokenStr.toLowerCase(), tag);
			sentenceParts.add(token);
			tokens.add(token);

			if (endOfSentenceDetector.isEndOfSentence(token)) {
				if (sentenceParts.size() > 1 && (float) unknownTokenCount / tokenCount < 0.5f) {
					sentences.add(new TaggedSentence(sentenceParts, tokens));
				} else {
					// badSentences++;
				}
				sentenceParts = new ArrayList<Object>();
				tokens = new ArrayList<Token>();
				tokenCount = 0;
				unknownTokenCount = 0;
			}
		}

		// System.out.println(badSentences + " of " + (sentences.size() +
		// badSentences));

		return sentences;
	}
	
	private String trimLeft(String str) {
		int i;
		for (i = 0; i < str.length(); i++) {
			char ch = str.charAt(i);
			if (!Character.isWhitespace(ch)) {
				break;
			}
		}
		if (i == 0) {
			return str;
		} else {
			return str.substring(i);
		}
	}

	public void setEndOfSentenceDetector(EndOfSentenceDetector endOfSentenceDetector) {
		this.endOfSentenceDetector = endOfSentenceDetector;
	}
}
